{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8f0c8a05-b3ce-4412-9662-201dc75006cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "允许抓取\n"
     ]
    }
   ],
   "source": [
    "import urllib.robotparser\n",
    "import requests\n",
    "#读取robots.txt文件\n",
    "rp = urllib.robotparser.RobotFileParser()\n",
    "rp.set_url(\"https://item.taobao.com/robots.txt\")\n",
    "rp.read()\n",
    "useragent='Googlebot'    #模拟Googlebot，能生成文件\n",
    "#useragent='Baiduspider'    #模拟Baiduspider，不能生成文件\n",
    "url='https://item.taobao.com/item.htm?spm=a219r.lm897.14.38.5d2346e28rO73l&id=522811099442&ns=1&abbucket=7'\n",
    "if rp.can_fetch(useragent, url):\n",
    "  print(\"允许抓取\")\n",
    "  file=requests.get(url)\n",
    "  data=file.content         #读取全部\n",
    "  fb=open(\"C:/Users/86151/Desktop/爬虫/taobao\",\"wb\")    #将爬取的网页保存在本地\n",
    "  fb.write(data)\n",
    "  fb.close()\n",
    "else:\n",
    "  print(\"不允许抓取\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c2d6cb2f-b964-4f4d-957e-5d3aea08607e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "爬取成功！已保存为 taobao_homepage.html\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "# 1. 核心请求头\n",
    "headers = {\n",
    "'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',   \n",
    "'Cookie': 'https://item.taobao.com/item.htm?spm=a219r.lm897.14.38.5d2346e28rO73l&id=522811099442&ns=1&abbucket=7', \n",
    "'Referer': 'https://www.taobao.com/',  \n",
    "'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n",
    "'Accept-Language': 'zh-CN,zh;q=0.9' \n",
    " }\n",
    "# 2. 请求配置\n",
    "url = 'https://www.taobao.com/'\n",
    "timeout = 15  # 延长超时时间，应对网络波动\n",
    "try: \n",
    "    # 3. 发送GET请求    \n",
    "    response = requests.get(\n",
    "        url,     \n",
    "        headers=headers,\n",
    "        timeout=timeout,\n",
    "        verify=True  # 验证SSL证书，避免安全风险\n",
    "  )   \n",
    "# 4. 自动适配编码（解决中文乱码）   \n",
    "    if response.encoding == 'ISO-8859-1':\n",
    "       response.encoding = response.apparent_encoding  \n",
    "       # 用chardet自动检测编码  \n",
    "# 5. 验证请求与保存   \n",
    "    if response.status_code == 200:   # 检查是否为正常首页（含\"淘宝\"关键词则说明未被拦截）       \n",
    "       if '淘宝' in response.text:         \n",
    "           with open('C:/Users/86151/Desktop/爬虫/taobao_homepage.html', 'w', encoding=response.encoding) as f:\n",
    "               f.write(response.text)          \n",
    "               print(\"爬取成功！已保存为 taobao_homepage.html\")        \n",
    "       else:          \n",
    "           print(\"内容异常，可能仍被拦截\")  \n",
    "    else: \n",
    "        print(f\"请求失败，状态码：{response.status_code}\")\n",
    "except requests.exceptions.ConnectTimeout:\n",
    "    print(\"连接超时，请检查网络\")\n",
    "except requests.exceptions.SSLError:   \n",
    "    print(\"SSL证书错误，可尝试将verify改为False\")\n",
    "except Exception as e:  \n",
    "    print(f\"其他错误：{str(e)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd9d9de2-baf1-40ac-8b81-3446e974fbad",
   "metadata": {},
   "outputs": [],
   "source": [
    "url=https://www.gov.cn/yaowen/liebiao/202502/content_7004472.htm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb50405a-3c3f-4351-8622-d238aa6042d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests<br/><br/>\n",
    "# 1. 请求头设置<br/>\n",
    "headers = {<br/>    \n",
    "Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',<br/>    \n",
    "'Accept-Language': 'zh-CN,zh;q=0.9',<br/>    \n",
    "'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'<br/>}<br/><br/>\n",
    "# 2. 请求配置<br/>\n",
    "url = 'https://www.taobao.com/'<br/>\n",
    "timeout = 15  # 延长超时时间，应对网络波动<br/><br/>\n",
    "try:<br/>  \n",
    "    # 3. 发送 GET 请求<br/> \n",
    "response = requests.get(<br/>   \n",
    "url,<br/>        \n",
    "headers=headers,<br/>      \n",
    "timeout=timeout,<br/>       \n",
    "verify=True  # 验证 SSL 证书，避免安全风险<br/>   \n",
    ")<br/><br/>   \n",
    "# 4. 自动适配编码（解决中文乱码）<br/>  \n",
    "if response.encoding == 'ISO-8859-1':<br/>     \n",
    "    response.encoding = response.apparent_encoding<br/><br/>    \n",
    "# 5. 验证请求与保存<br/> \n",
    "if response.status_code == 200:<br/>    \n",
    "    if '淘宝' in response.text:<br/>        \n",
    "        with open('taobao_homepage.html', 'w', encoding=response.encoding) as f:<br/>   \n",
    "            f.write(response.text)<br/>        \n",
    "print(\"爬取成功！已保存为 taobao_homepage.html\")<br/>     \n",
    "else:<br/>         \n",
    "print(\"内容异常，可能仍被拦截\")<br/>   \n",
    "else:<br/>    \n",
    "print(f\"请求失败，状态码：{response.status_code}\")<br/><br/>\n",
    "except requests.exceptions.ConnectTimeout:<br/>   \n",
    "print(\"连接超时，请检查网络\")<br/>\n",
    "except requests.exceptions.SSLError:<br/>    \n",
    "print(\"SSL 证书错误，可尝试将 verify 改为 False\")<br/>\n",
    "except Exception as e:<br/>    \n",
    "print(f\"其他错误：{str(e)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "d0262d40-7048-4936-a381-0be3a4f5fd9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "服务器返回的所有Cookies：\n",
      "<RequestsCookieJar[]>\n",
      "\n",
      "指定Cookie的值（例如获取'JSESSIONID'）：\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "# 目标网址（以政府网为例）\n",
    "url = \"https://www.gov.cn/\"\n",
    "headers = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36\"\n",
    "}\n",
    "# 发送GET请求\n",
    "response = requests.get(url, headers=headers)\n",
    "# 查看所有Cookies（返回CookieJar对象）\n",
    "print(\"服务器返回的所有Cookies：\")\n",
    "print(response.cookies)\n",
    "# 查看单个Cookie的值（按Cookie的name获取）\n",
    "print(\"\\n指定Cookie的值（例如获取'JSESSIONID'）：\")\n",
    "print(response.cookies.get(\"JSESSIONID\"))  # 把\"JSESSIONID\"换成实际要查的Cookie名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "81f844e3-2bea-4f92-ad67-2facbca4d0c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "服务器返回的所有Cookies：\n",
      "<RequestsCookieJar[]>\n",
      "\n",
      "指定Cookie的值（例如获取'JSESSIONID'）：\n",
      "None\n",
      "政府网服务器返回的所有Cookie：\n",
      "--------------------------------------------------\n",
      "\n",
      "可直接复制到爬虫headers里的Cookie字符串：\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "# 目标网址（以政府网为例）\n",
    "url = \"https://www.gov.cn/\"\n",
    "headers = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36\"\n",
    "}\n",
    "# 发送GET请求\n",
    "response = requests.get(url, headers=headers)\n",
    "# 查看所有Cookies（返回CookieJar对象）\n",
    "print(\"服务器返回的所有Cookies：\")\n",
    "print(response.cookies)\n",
    "# 查看单个Cookie的值（按Cookie的name获取）\n",
    "print(\"\\n指定Cookie的值（例如获取'JSESSIONID'）：\")\n",
    "print(response.cookies.get(\"JSESSIONID\"))  # 把\"JSESSIONID\"换成实际要查的Cookie名\n",
    "\n",
    "import requests# 目标政府网网址\n",
    "url = \"https://www.gov.cn/\"\n",
    "headers = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36\"\n",
    "}\n",
    "# 发送请求并获取Cookies\n",
    "response = requests.get(url, headers=headers)\n",
    "# 自动遍历所有Cookie，打印“名称+值”\n",
    "print(\"政府网服务器返回的所有Cookie：\")\n",
    "print(\"-\" * 50)  # 分隔线，让结果更清晰\n",
    "for cookie in response.cookies:\n",
    "    # 逐个输出Cookie的名称和对应值\n",
    "    print(f\"Cookie名称：{cookie.name}\")\n",
    "    print(f\"Cookie值：{cookie.value}\")\n",
    "    print(\"-\" * 50)\n",
    "    # 额外：直接生成请求头可用的Cookie字符串（复制就能贴进爬虫代码）\n",
    "cookie_str = \"; \".join([f\"{cookie.name}={cookie.value}\" for cookie in response.cookies])\n",
    "print(\"\\n可直接复制到爬虫headers里的Cookie字符串：\")\n",
    "print(cookie_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "1f61a1e6-6e2c-458b-98fb-f4e29e1b8a0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "robots.txt 允许爬取该页面\n",
      "爬取成功，已保存为 gov.html\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import urllib.robotparser\n",
    "# 1. 读取 robots.txt\n",
    "rp = urllib.robotparser.RobotFileParser()\n",
    "rp.set_url(\"https://www.gov.cn/robots.txt\")\n",
    "rp.read()\n",
    "# 2. 检查是否允许爬取目标页面\n",
    "url_to_crawl = \"https://www.gov.cn/yaowen/liebiao/202502/content_7004472.htm\"\n",
    "if rp.can_fetch(\"*\", url_to_crawl):\n",
    "    print(\"robots.txt 允许爬取该页面\")\n",
    "else:\n",
    "    print(\"robots.txt 禁止爬取该页面，停止操作\")\n",
    "    exit()\n",
    "    # 3. 设置请求头，模拟浏览器\n",
    "headers = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',\n",
    "    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n",
    "    'Accept-Language': 'zh-CN,zh;q=0.9'\n",
    "    'Cookie' 'https://www.gov.cn/D3D83DD6216CE942DFBE33961F3230FF'\n",
    "    'Accept-Encoding' 'identity'\n",
    "}\n",
    "# 4. 发送 GET 请求\n",
    "try:\n",
    "    response = requests.get(url_to_crawl, headers=headers, timeout=10, verify=True)\n",
    "    response.raise_for_status()  # 如果状态码不是 200，引发 HTTPError 异常\n",
    "    # 5. 保存网页内容\n",
    "    with open(\"C:/Users/86151/Desktop/爬虫/gov.html\", \"w\", encoding=\"utf-8\") as f:\n",
    "        f.write(response.text)\n",
    "        print(\"爬取成功，已保存为 gov.html\")\n",
    "except requests.exceptions.RequestException as e:\n",
    "    print(f\"爬取失败：{e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c439941a-3acd-4c05-bcdc-77ee9b5cbac0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:base] *",
   "language": "python",
   "name": "conda-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
